{
 "metadata": {
  "kernelspec": {
   "language": "python",
   "display_name": "Python 3",
   "name": "python3"
  },
  "language_info": {
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "version": "3.6.4",
   "file_extension": ".py",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "name": "python",
   "mimetype": "text/x-python"
  }
 },
 "nbformat_minor": 4,
 "nbformat": 4,
 "cells": [
  {
   "cell_type": "code",
   "source": [
    "!pip install datasets \n",
    "!pip install transformers\n",
    "!pip install sentencepiece\n",
    "!pip install -U scikit-learn"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:25:13.580788Z",
     "iopub.execute_input": "2022-12-08T02:25:13.581209Z",
     "iopub.status.idle": "2022-12-08T02:25:43.579092Z",
     "shell.execute_reply.started": "2022-12-08T02:25:13.581174Z",
     "shell.execute_reply": "2022-12-08T02:25:43.577875Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import KFold, StratifiedKFold, StratifiedGroupKFold\n",
    "import shutil\n",
    "import time\n",
    "import gc\n",
    "import random\n",
    "import math\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "import transformers\n",
    "from transformers import TrainingArguments, Trainer, DataCollatorForWholeWordMask\n",
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer, AutoConfig, AutoModel\n",
    "from torch import nn\n",
    "from torch.optim import Adam, SGD, AdamW\n",
    "from transformers import get_linear_schedule_with_warmup, get_cosine_schedule_with_warmup\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\""
   ],
   "metadata": {
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "execution": {
     "iopub.status.busy": "2022-12-08T02:27:42.250417Z",
     "iopub.execute_input": "2022-12-08T02:27:42.250952Z",
     "iopub.status.idle": "2022-12-08T02:27:50.826888Z",
     "shell.execute_reply.started": "2022-12-08T02:27:42.250833Z",
     "shell.execute_reply": "2022-12-08T02:27:50.825188Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import transformers\n",
    "\n",
    "print(transformers.__version__)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:27:50.832500Z",
     "iopub.execute_input": "2022-12-08T02:27:50.834772Z",
     "iopub.status.idle": "2022-12-08T02:27:50.845711Z",
     "shell.execute_reply.started": "2022-12-08T02:27:50.834728Z",
     "shell.execute_reply": "2022-12-08T02:27:50.844580Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "from datasets import load_metric\n",
    "metric = load_metric('glue', 'stsb')\n",
    "metric"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:27:50.850777Z",
     "iopub.execute_input": "2022-12-08T02:27:50.851461Z",
     "iopub.status.idle": "2022-12-08T02:27:51.836124Z",
     "shell.execute_reply.started": "2022-12-08T02:27:50.851424Z",
     "shell.execute_reply": "2022-12-08T02:27:51.835026Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "from transformers import AutoTokenizer\n",
    "# model_checkpoint = 'microsoft/deberta-v3-small'\n",
    "# model_checkpoint='anferico/bert-for-patents'\n",
    "model_checkpoint='bert-base-uncased'\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_checkpoint)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:08.842537Z",
     "iopub.execute_input": "2022-12-08T02:29:08.842943Z",
     "iopub.status.idle": "2022-12-08T02:29:15.719646Z",
     "shell.execute_reply.started": "2022-12-08T02:29:08.842910Z",
     "shell.execute_reply": "2022-12-08T02:29:15.718764Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df=pd.read_csv('../input/us-patent-phrase-to-phrase-matching/train.csv')\n",
    "df_title=pd.read_csv('../input/cpc-codes/titles.csv') "
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:15.721541Z",
     "iopub.execute_input": "2022-12-08T02:29:15.721999Z",
     "iopub.status.idle": "2022-12-08T02:29:16.196895Z",
     "shell.execute_reply.started": "2022-12-08T02:29:15.721961Z",
     "shell.execute_reply": "2022-12-08T02:29:16.195912Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.198609Z",
     "iopub.execute_input": "2022-12-08T02:29:16.199311Z",
     "iopub.status.idle": "2022-12-08T02:29:16.218332Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.199271Z",
     "shell.execute_reply": "2022-12-08T02:29:16.217420Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df_title"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.221716Z",
     "iopub.execute_input": "2022-12-08T02:29:16.222585Z",
     "iopub.status.idle": "2022-12-08T02:29:16.242347Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.222557Z",
     "shell.execute_reply": "2022-12-08T02:29:16.241555Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df = df.merge(df_title, how='left', left_on='context', right_on='code')\n",
    "df = df[['id', 'anchor', 'target', 'context', 'score', 'title']]"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.244665Z",
     "iopub.execute_input": "2022-12-08T02:29:16.245271Z",
     "iopub.status.idle": "2022-12-08T02:29:16.335312Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.245234Z",
     "shell.execute_reply": "2022-12-08T02:29:16.334343Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "kf = StratifiedGroupKFold(n_splits=5, shuffle=True, random_state=42)\n",
    "df['fold'] = -1\n",
    "for f, (t_, v_) in enumerate(kf.split(X=df, y=df['anchor'], groups=df['anchor'])):\n",
    "    df.loc[v_, 'fold'] = f "
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.336860Z",
     "iopub.execute_input": "2022-12-08T02:29:16.337260Z",
     "iopub.status.idle": "2022-12-08T02:29:16.865910Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.337222Z",
     "shell.execute_reply": "2022-12-08T02:29:16.864807Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.867254Z",
     "iopub.execute_input": "2022-12-08T02:29:16.868266Z",
     "iopub.status.idle": "2022-12-08T02:29:16.887686Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.868225Z",
     "shell.execute_reply": "2022-12-08T02:29:16.886772Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "df['input'] = df['anchor'] + tokenizer.sep_token + df['title'].apply(str.lower)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.889216Z",
     "iopub.execute_input": "2022-12-08T02:29:16.889578Z",
     "iopub.status.idle": "2022-12-08T02:29:16.912027Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.889543Z",
     "shell.execute_reply": "2022-12-08T02:29:16.911100Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "import torch\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "class TrainDataset(Dataset):\n",
    "    def __init__(self, df):\n",
    "        self.inputs = df['input'].values.astype(str)\n",
    "        self.targets = df['target'].values.astype(str)\n",
    "        self.label = df['score'].values\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.inputs)\n",
    "\n",
    "    def __getitem__(self, item):\n",
    "        inputs = self.inputs[item]\n",
    "        targets = self.targets[item]\n",
    "        label = self.label[item]\n",
    "        \n",
    "        \n",
    "        inputs = tokenizer(inputs,    \n",
    "                    targets,\n",
    "                    max_length=64,\n",
    "                    padding='max_length',\n",
    "                    truncation=True )\n",
    "        return {**inputs,\n",
    "               'labels':torch.as_tensor(label, dtype=torch.float) }"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.913497Z",
     "iopub.execute_input": "2022-12-08T02:29:16.913866Z",
     "iopub.status.idle": "2022-12-08T02:29:16.923760Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.913832Z",
     "shell.execute_reply": "2022-12-08T02:29:16.922781Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "from transformers import AutoModelForSequenceClassification, TrainingArguments, Trainer\n",
    "\n",
    "num_labels = 1 \n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:16.966048Z",
     "iopub.execute_input": "2022-12-08T02:29:16.966314Z",
     "iopub.status.idle": "2022-12-08T02:29:27.857118Z",
     "shell.execute_reply.started": "2022-12-08T02:29:16.966290Z",
     "shell.execute_reply": "2022-12-08T02:29:27.856127Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "metric_name = \"pearson\"\n",
    "model_name = model_checkpoint.split(\"/\")[-1]\n",
    "batch_size = 200\n",
    "args = TrainingArguments(\n",
    "    f\"{model_name}-finetuned\",\n",
    "    evaluation_strategy = \"epoch\",\n",
    "    save_strategy = \"epoch\",\n",
    "    learning_rate=2e-5,\n",
    "    per_device_train_batch_size=batch_size,\n",
    "    per_device_eval_batch_size=batch_size,\n",
    "    num_train_epochs=5,\n",
    "    weight_decay=0.01,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model=metric_name,\n",
    "    save_total_limit=1,\n",
    ")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:27.859566Z",
     "iopub.execute_input": "2022-12-08T02:29:27.860010Z",
     "iopub.status.idle": "2022-12-08T02:29:27.873518Z",
     "shell.execute_reply.started": "2022-12-08T02:29:27.859970Z",
     "shell.execute_reply": "2022-12-08T02:29:27.871946Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    predictions, labels = eval_pred\n",
    "    return metric.compute(predictions=predictions, references=labels)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:27.875589Z",
     "iopub.execute_input": "2022-12-08T02:29:27.876069Z",
     "iopub.status.idle": "2022-12-08T02:29:27.881881Z",
     "shell.execute_reply.started": "2022-12-08T02:29:27.876029Z",
     "shell.execute_reply": "2022-12-08T02:29:27.880830Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "train_dataset = TrainDataset(df[df['fold'] != 0])\n",
    "val_dataset = TrainDataset(df[df['fold'] == 0])\n",
    "trainer = Trainer(\n",
    "    model,\n",
    "    args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics,\n",
    "\n",
    ")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:27.884676Z",
     "iopub.execute_input": "2022-12-08T02:29:27.885105Z",
     "iopub.status.idle": "2022-12-08T02:29:28.064653Z",
     "shell.execute_reply.started": "2022-12-08T02:29:27.885070Z",
     "shell.execute_reply": "2022-12-08T02:29:28.063488Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "trainer.train()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:29:28.066346Z",
     "iopub.execute_input": "2022-12-08T02:29:28.066728Z",
     "iopub.status.idle": "2022-12-08T02:43:53.273335Z",
     "shell.execute_reply.started": "2022-12-08T02:29:28.066692Z",
     "shell.execute_reply": "2022-12-08T02:43:53.271929Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "# CUDA_VISBLE_DEVICE=0,1,2,3 python run_train.py"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:16:55.696218Z",
     "iopub.status.idle": "2022-12-08T02:16:55.697470Z",
     "shell.execute_reply.started": "2022-12-08T02:16:55.697188Z",
     "shell.execute_reply": "2022-12-08T02:16:55.697213Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "trainer.evaluate()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:59:36.026725Z",
     "iopub.execute_input": "2022-12-08T02:59:36.027448Z",
     "iopub.status.idle": "2022-12-08T02:59:49.086361Z",
     "shell.execute_reply.started": "2022-12-08T02:59:36.027408Z",
     "shell.execute_reply": "2022-12-08T02:59:49.085443Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "! pip install optuna\n",
    "! pip install ray[tune]"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T02:59:49.088080Z",
     "iopub.execute_input": "2022-12-08T02:59:49.088429Z",
     "iopub.status.idle": "2022-12-08T03:00:09.141410Z",
     "shell.execute_reply.started": "2022-12-08T02:59:49.088395Z",
     "shell.execute_reply": "2022-12-08T03:00:09.140221Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "def model_init():\n",
    "    return AutoModelForSequenceClassification.from_pretrained(model_checkpoint, num_labels=num_labels)"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T03:00:09.143545Z",
     "iopub.execute_input": "2022-12-08T03:00:09.143973Z",
     "iopub.status.idle": "2022-12-08T03:00:09.149788Z",
     "shell.execute_reply.started": "2022-12-08T03:00:09.143931Z",
     "shell.execute_reply": "2022-12-08T03:00:09.148861Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "trainer = Trainer(\n",
    "    model_init=model_init,\n",
    "    args=args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=val_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T03:00:09.152844Z",
     "iopub.execute_input": "2022-12-08T03:00:09.153457Z",
     "iopub.status.idle": "2022-12-08T03:00:11.761034Z",
     "shell.execute_reply.started": "2022-12-08T03:00:09.153420Z",
     "shell.execute_reply": "2022-12-08T03:00:11.760023Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "best_run = trainer.hyperparameter_search(n_trials=10, direction=\"maximize\")"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T03:00:11.762911Z",
     "iopub.execute_input": "2022-12-08T03:00:11.763585Z",
     "iopub.status.idle": "2022-12-08T04:04:09.778984Z",
     "shell.execute_reply.started": "2022-12-08T03:00:11.763547Z",
     "shell.execute_reply": "2022-12-08T04:04:09.778046Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "best_run"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T04:04:09.780615Z",
     "iopub.execute_input": "2022-12-08T04:04:09.781000Z",
     "iopub.status.idle": "2022-12-08T04:04:09.796932Z",
     "shell.execute_reply.started": "2022-12-08T04:04:09.780962Z",
     "shell.execute_reply": "2022-12-08T04:04:09.795863Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [
    "for n, v in best_run.hyperparameters.items():\n",
    "    setattr(trainer.args, n, v)\n",
    "\n",
    "trainer.train()"
   ],
   "metadata": {
    "execution": {
     "iopub.status.busy": "2022-12-08T04:04:09.798676Z",
     "iopub.execute_input": "2022-12-08T04:04:09.799053Z",
     "iopub.status.idle": "2022-12-08T04:18:38.595318Z",
     "shell.execute_reply.started": "2022-12-08T04:04:09.799019Z",
     "shell.execute_reply": "2022-12-08T04:18:38.593871Z"
    },
    "trusted": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  },
  {
   "cell_type": "code",
   "source": [],
   "metadata": {
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "execution_count": null,
   "outputs": []
  }
 ]
}
